AWSTemplateFormatVersion: '2010-09-09'
Description: >
  Old EBS Snapshot Cleanup Remediator V2.
  Purpose-specific, opt-in remediation module for identifying and optionally deleting
  old self-owned EBS snapshots that are not associated with current AMIs. Supports
  both approval-based cleanup and date-based batch cleanup with strong automatic
  exclusions for AWS Backup, DLM, and third-party backup products such as Clumio.

Parameters:
  ProjectName:
    Type: String
    Default: old-ebs-snapshot-cleanup-remediator
    Description: Base name used for created resources.

  TargetRegionsCsv:
    Type: String
    Default: all-enabled
    Description: Comma-separated AWS Regions to scan, or all-enabled to scan all enabled Regions.

  ExcludedRegionsCsv:
    Type: String
    Default: ap-northeast-3
    Description: Comma-separated AWS Regions to exclude from scanning/remediation.

  DryRun:
    Type: String
    Default: 'true'
    AllowedValues: ['true', 'false']
    Description: When true, only report what would be deleted. When false, delete eligible snapshots.

  MinimumSnapshotAgeDays:
    Type: Number
    Default: 180
    Description: Only snapshots older than this many days are considered candidates.

  RequireApprovalTag:
    Type: String
    Default: 'true'
    AllowedValues: ['true', 'false']
    Description: >
      When true, only delete candidate snapshots tagged with ApprovalTagKey=ApprovalTagValue.
      For large backlog cleanup, set false only after reviewing dry-run results and keeping backup exclusions enabled.

  ApprovalTagKey:
    Type: String
    Default: CostOptApproved
    Description: Tag key required when RequireApprovalTag is true.

  ApprovalTagValue:
    Type: String
    Default: 'true'
    Description: Tag value required when RequireApprovalTag is true.

  IgnoreTagKeysCsv:
    Type: String
    Default: CostOptIgnore,DoNotDelete
    Description: Comma-separated tag keys that prevent deletion when present, regardless of value.

  IgnoreTagPairsCsv:
    Type: String
    Default: CostOptIgnore=true,DoNotDelete=true
    Description: Comma-separated key=value tag pairs that prevent deletion when present.

  SkipAwsBackupSnapshots:
    Type: String
    Default: 'true'
    AllowedValues: ['true', 'false']
    Description: Skip snapshots that appear to be AWS Backup managed based on tags or description.

  SkipDlmSnapshots:
    Type: String
    Default: 'true'
    AllowedValues: ['true', 'false']
    Description: Skip snapshots that appear to be Data Lifecycle Manager managed based on tags or description.

  SkipThirdPartyBackupSnapshots:
    Type: String
    Default: 'true'
    AllowedValues: ['true', 'false']
    Description: >
      Skip snapshots that appear to be managed by third-party backup products such as
      Clumio, Veeam, Commvault, Rubrik, Druva, or Cohesity.

  ThirdPartyBackupMarkersCsv:
    Type: String
    Default: Clumio,Veeam,Commvault,Rubrik,Druva,Cohesity,BackupID,BackupAssetID,TaskID,OrgID,Vendor=Clumio,OperationName=Clumio Backup Snapshot,OperationType=EC2 Backup,Clumio Backup Snapshot
    Description: Comma-separated text markers checked against snapshot description and tags when SkipThirdPartyBackupSnapshots is true.

  MaxSnapshotsPerRun:
    Type: Number
    Default: 100
    Description: Safety limit. 0 means no limit. Recommended batch cleanup value is 100 or 500.

  ContinueOnAccessDenied:
    Type: String
    Default: 'true'
    AllowedValues: ['true', 'false']
    Description: Continue scanning other Regions/resources when blocked by IAM/SCP.

  ScheduleEnabled:
    Type: String
    Default: 'false'
    AllowedValues: ['true', 'false']
    Description: Whether to create an enabled EventBridge schedule for recurring runs.

  ScheduleExpression:
    Type: String
    Default: rate(30 days)
    Description: EventBridge schedule expression used when ScheduleEnabled is true.

Conditions:
  IsScheduleEnabled: !Equals [!Ref ScheduleEnabled, 'true']

Resources:
  RemediatorRole:
    Type: AWS::IAM::Role
    Properties:
      RoleName: !Sub '${ProjectName}-${AWS::AccountId}-${AWS::Region}-role'
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              Service: lambda.amazonaws.com
            Action: sts:AssumeRole
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole
      Policies:
        - PolicyName: OldEbsSnapshotCleanupPolicy
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Sid: DiscoverRegions
                Effect: Allow
                Action:
                  - ec2:DescribeRegions
                Resource: '*'
              - Sid: ReadSnapshotsImagesAndTags
                Effect: Allow
                Action:
                  - ec2:DescribeSnapshots
                  - ec2:DescribeImages
                  - ec2:DescribeTags
                Resource: '*'
              - Sid: DeleteSnapshots
                Effect: Allow
                Action:
                  - ec2:DeleteSnapshot
                Resource: '*'

  RemediatorFunction:
    Type: AWS::Lambda::Function
    Properties:
      FunctionName: !Ref ProjectName
      Description: Purpose-specific remediation Lambda for deleting eligible old EBS snapshots.
      Runtime: python3.12
      Handler: index.handler
      Timeout: 900
      MemorySize: 512
      Role: !GetAtt RemediatorRole.Arn
      Environment:
        Variables:
          TARGET_REGIONS_CSV: !Ref TargetRegionsCsv
          EXCLUDED_REGIONS_CSV: !Ref ExcludedRegionsCsv
          DRY_RUN: !Ref DryRun
          MINIMUM_SNAPSHOT_AGE_DAYS: !Ref MinimumSnapshotAgeDays
          REQUIRE_APPROVAL_TAG: !Ref RequireApprovalTag
          APPROVAL_TAG_KEY: !Ref ApprovalTagKey
          APPROVAL_TAG_VALUE: !Ref ApprovalTagValue
          IGNORE_TAG_KEYS_CSV: !Ref IgnoreTagKeysCsv
          IGNORE_TAG_PAIRS_CSV: !Ref IgnoreTagPairsCsv
          SKIP_AWS_BACKUP_SNAPSHOTS: !Ref SkipAwsBackupSnapshots
          SKIP_DLM_SNAPSHOTS: !Ref SkipDlmSnapshots
          SKIP_THIRD_PARTY_BACKUP_SNAPSHOTS: !Ref SkipThirdPartyBackupSnapshots
          THIRD_PARTY_BACKUP_MARKERS_CSV: !Ref ThirdPartyBackupMarkersCsv
          MAX_SNAPSHOTS_PER_RUN: !Ref MaxSnapshotsPerRun
          CONTINUE_ON_ACCESS_DENIED: !Ref ContinueOnAccessDenied
      Code:
        ZipFile: |
          import json
          import os
          import traceback
          from datetime import datetime, timezone

          import boto3
          from botocore.exceptions import ClientError

          def utc_now():
              return datetime.now(timezone.utc)

          def utc_now_iso():
              return utc_now().isoformat()

          def env_bool(name, default=False):
              value = os.environ.get(name, str(default)).strip().lower()
              return value in ('1', 'true', 'yes', 'y')

          def env_int(name, default=0):
              try:
                  return int(os.environ.get(name, str(default)).strip())
              except Exception:
                  return default

          def split_csv(value):
              if value is None:
                  return []
              return [item.strip() for item in str(value).split(',') if item.strip()]

          def parse_tag_pairs(value):
              pairs = {}
              for item in split_csv(value):
                  if '=' not in item:
                      continue
                  key, val = item.split('=', 1)
                  key = key.strip()
                  val = val.strip()
                  if key:
                      pairs[key] = val
              return pairs

          def tags_to_dict(tags):
              result = {}
              for tag in tags or []:
                  key = tag.get('Key')
                  if key:
                      result[key] = tag.get('Value', '')
              return result

          def get_enabled_regions(excluded_regions):
              requested = os.environ.get('TARGET_REGIONS_CSV', 'all-enabled').strip()
              if requested and requested.lower() != 'all-enabled':
                  regions = split_csv(requested)
              else:
                  ec2 = boto3.client('ec2', region_name=os.environ.get('AWS_REGION', 'us-east-1'))
                  response = ec2.describe_regions(AllRegions=False)
                  regions = sorted([r['RegionName'] for r in response.get('Regions', [])])
              return [r for r in regions if r not in excluded_regions]

          def has_ignore_tag(tag_map, ignore_tag_keys, ignore_tag_pairs):
              for key in ignore_tag_keys:
                  if key in tag_map:
                      return True, 'ignore_tag_key=' + key
              for key, value in ignore_tag_pairs.items():
                  if tag_map.get(key) == value:
                      return True, 'ignore_tag_pair=' + key + '=' + value
              return False, ''

          def has_approval_tag(tag_map, settings):
              if not settings['require_approval_tag']:
                  return True
              return tag_map.get(settings['approval_tag_key']) == settings['approval_tag_value']

          def snapshot_age_days(snapshot):
              start_time = snapshot.get('StartTime')
              if not start_time:
                  return None
              if start_time.tzinfo is None:
                  start_time = start_time.replace(tzinfo=timezone.utc)
              return (utc_now() - start_time).days

          def get_ami_snapshot_ids(ec2):
              snapshot_ids = set()
              paginator = ec2.get_paginator('describe_images')
              for page in paginator.paginate(Owners=['self']):
                  for image in page.get('Images', []):
                      for mapping in image.get('BlockDeviceMappings', []):
                          ebs = mapping.get('Ebs') or {}
                          sid = ebs.get('SnapshotId')
                          if sid:
                              snapshot_ids.add(sid)
              return snapshot_ids

          def searchable_snapshot_text(snapshot, tag_map):
              parts = [(snapshot.get('Description') or '').lower()]
              for key, value in tag_map.items():
                  parts.append((str(key) + '=' + str(value)).lower())
                  parts.append(str(key).lower())
                  parts.append(str(value).lower())
              return ' '.join(parts)

          def appears_aws_backup_managed(snapshot, tag_map):
              text = searchable_snapshot_text(snapshot, tag_map)
              markers = ['awsbackup', 'aws backup', 'aws:backup', 'backup plan', 'recovery point']
              return any(marker in text for marker in markers)

          def appears_dlm_managed(snapshot, tag_map):
              text = searchable_snapshot_text(snapshot, tag_map)
              if 'created by createimage' in text:
                  return False
              markers = ['aws:dlm', 'dlm', 'data lifecycle manager']
              return any(marker in text for marker in markers)

          def appears_third_party_backup_managed(snapshot, tag_map, markers):
              text = searchable_snapshot_text(snapshot, tag_map)
              for marker in markers:
                  if marker.lower() in text:
                      return True
              return False

          def scan_region(region, settings):
              ec2 = boto3.client('ec2', region_name=region)
              result = {
                  'region': region,
                  'evaluated': 0,
                  'old_snapshots': 0,
                  'would_delete': 0,
                  'deleted': 0,
                  'skipped_too_new': 0,
                  'skipped_ami_associated': 0,
                  'skipped_aws_backup_managed': 0,
                  'skipped_dlm_managed': 0,
                  'skipped_third_party_backup_managed': 0,
                  'skipped_missing_approval_tag': 0,
                  'skipped_ignore_tag': 0,
                  'access_denied': 0,
                  'failed': 0,
                  'failures': [],
                  'candidates': [],
                  'deleted_snapshots': []
              }

              try:
                  ami_snapshot_ids = get_ami_snapshot_ids(ec2)
              except ClientError as e:
                  code = e.response.get('Error', {}).get('Code', 'Unknown')
                  message = e.response.get('Error', {}).get('Message', str(e))
                  result['failures'].append({'operation': 'DescribeImages', 'error_code': code, 'error_message': message})
                  if code in ('AccessDenied', 'AccessDeniedException', 'UnauthorizedOperation') or 'explicit deny' in message.lower():
                      result['access_denied'] += 1
                  else:
                      result['failed'] += 1
                  if not settings['continue_on_access_denied']:
                      raise
                  return result

              try:
                  paginator = ec2.get_paginator('describe_snapshots')
                  pages = paginator.paginate(OwnerIds=['self'])
              except ClientError as e:
                  code = e.response.get('Error', {}).get('Code', 'Unknown')
                  message = e.response.get('Error', {}).get('Message', str(e))
                  result['failures'].append({'operation': 'DescribeSnapshots', 'error_code': code, 'error_message': message})
                  if code in ('AccessDenied', 'AccessDeniedException', 'UnauthorizedOperation') or 'explicit deny' in message.lower():
                      result['access_denied'] += 1
                  else:
                      result['failed'] += 1
                  if not settings['continue_on_access_denied']:
                      raise
                  return result

              for page in pages:
                  for snapshot in page.get('Snapshots', []):
                      if settings['max_snapshots_per_run'] and (result['would_delete'] + result['deleted']) >= settings['max_snapshots_per_run']:
                          result['stopped_due_to_max_snapshots_per_run'] = True
                          return result

                      result['evaluated'] += 1
                      snapshot_id = snapshot.get('SnapshotId')
                      age_days = snapshot_age_days(snapshot)
                      tag_map = tags_to_dict(snapshot.get('Tags', []))
                      volume_id = snapshot.get('VolumeId')
                      volume_size = snapshot.get('VolumeSize')
                      description = snapshot.get('Description')

                      if age_days is None or age_days < settings['minimum_snapshot_age_days']:
                          result['skipped_too_new'] += 1
                          continue

                      result['old_snapshots'] += 1

                      if snapshot_id in ami_snapshot_ids:
                          result['skipped_ami_associated'] += 1
                          continue

                      if settings['skip_aws_backup_snapshots'] and appears_aws_backup_managed(snapshot, tag_map):
                          result['skipped_aws_backup_managed'] += 1
                          continue

                      if settings['skip_dlm_snapshots'] and appears_dlm_managed(snapshot, tag_map):
                          result['skipped_dlm_managed'] += 1
                          continue

                      if settings['skip_third_party_backup_snapshots'] and appears_third_party_backup_managed(snapshot, tag_map, settings['third_party_backup_markers']):
                          result['skipped_third_party_backup_managed'] += 1
                          continue

                      ignore, ignore_reason = has_ignore_tag(tag_map, settings['ignore_tag_keys'], settings['ignore_tag_pairs'])
                      if ignore:
                          result['skipped_ignore_tag'] += 1
                          result['candidates'].append({
                              'snapshot_id': snapshot_id,
                              'age_days': age_days,
                              'volume_id': volume_id,
                              'volume_size_gib': volume_size,
                              'action': 'skipped_ignore_tag',
                              'reason': ignore_reason,
                              'tags': tag_map
                          })
                          continue

                      if not has_approval_tag(tag_map, settings):
                          result['skipped_missing_approval_tag'] += 1
                          result['candidates'].append({
                              'snapshot_id': snapshot_id,
                              'age_days': age_days,
                              'volume_id': volume_id,
                              'volume_size_gib': volume_size,
                              'action': 'skipped_missing_approval_tag',
                              'required_tag': settings['approval_tag_key'] + '=' + settings['approval_tag_value'],
                              'description': description,
                              'tags': tag_map
                          })
                          continue

                      candidate = {
                          'snapshot_id': snapshot_id,
                          'age_days': age_days,
                          'volume_id': volume_id,
                          'volume_size_gib': volume_size,
                          'description': description,
                          'tags': tag_map,
                          'action': 'would_delete' if settings['dry_run'] else 'delete'
                      }

                      if settings['dry_run']:
                          result['would_delete'] += 1
                          result['candidates'].append(candidate)
                          continue

                      try:
                          ec2.delete_snapshot(SnapshotId=snapshot_id)
                          result['deleted'] += 1
                          result['deleted_snapshots'].append(candidate)
                      except ClientError as e:
                          code = e.response.get('Error', {}).get('Code', 'Unknown')
                          message = e.response.get('Error', {}).get('Message', str(e))
                          failure = {'snapshot_id': snapshot_id, 'error_code': code, 'error_message': message}
                          if code in ('AccessDenied', 'AccessDeniedException', 'UnauthorizedOperation') or 'explicit deny' in message.lower():
                              result['access_denied'] += 1
                          else:
                              result['failed'] += 1
                          result['failures'].append(failure)
                          if not settings['continue_on_access_denied']:
                              raise

              return result

          def handler(event, context):
              settings = {
                  'excluded_regions': split_csv(os.environ.get('EXCLUDED_REGIONS_CSV', '')),
                  'dry_run': env_bool('DRY_RUN', True),
                  'minimum_snapshot_age_days': env_int('MINIMUM_SNAPSHOT_AGE_DAYS', 180),
                  'require_approval_tag': env_bool('REQUIRE_APPROVAL_TAG', True),
                  'approval_tag_key': os.environ.get('APPROVAL_TAG_KEY', 'CostOptApproved'),
                  'approval_tag_value': os.environ.get('APPROVAL_TAG_VALUE', 'true'),
                  'ignore_tag_keys': split_csv(os.environ.get('IGNORE_TAG_KEYS_CSV', '')),
                  'ignore_tag_pairs': parse_tag_pairs(os.environ.get('IGNORE_TAG_PAIRS_CSV', '')),
                  'skip_aws_backup_snapshots': env_bool('SKIP_AWS_BACKUP_SNAPSHOTS', True),
                  'skip_dlm_snapshots': env_bool('SKIP_DLM_SNAPSHOTS', True),
                  'skip_third_party_backup_snapshots': env_bool('SKIP_THIRD_PARTY_BACKUP_SNAPSHOTS', True),
                  'third_party_backup_markers': split_csv(os.environ.get('THIRD_PARTY_BACKUP_MARKERS_CSV', '')),
                  'max_snapshots_per_run': env_int('MAX_SNAPSHOTS_PER_RUN', 100),
                  'continue_on_access_denied': env_bool('CONTINUE_ON_ACCESS_DENIED', True),
              }

              if isinstance(event, dict):
                  if 'dry_run' in event:
                      settings['dry_run'] = bool(event['dry_run'])
                  if 'require_approval_tag' in event:
                      settings['require_approval_tag'] = bool(event['require_approval_tag'])
                  if 'minimum_snapshot_age_days' in event:
                      settings['minimum_snapshot_age_days'] = int(event['minimum_snapshot_age_days'])
                  if 'max_snapshots_per_run' in event:
                      settings['max_snapshots_per_run'] = int(event['max_snapshots_per_run'])
                  if 'skip_third_party_backup_snapshots' in event:
                      settings['skip_third_party_backup_snapshots'] = bool(event['skip_third_party_backup_snapshots'])
                  if 'target_regions' in event and event['target_regions']:
                      os.environ['TARGET_REGIONS_CSV'] = ','.join(event['target_regions'])
                  if 'excluded_regions' in event and event['excluded_regions']:
                      settings['excluded_regions'] = list(event['excluded_regions'])

              summary = {
                  'started_at': utc_now_iso(),
                  'dry_run': settings['dry_run'],
                  'minimum_snapshot_age_days': settings['minimum_snapshot_age_days'],
                  'require_approval_tag': settings['require_approval_tag'],
                  'approval_tag': settings['approval_tag_key'] + '=' + settings['approval_tag_value'],
                  'skip_aws_backup_snapshots': settings['skip_aws_backup_snapshots'],
                  'skip_dlm_snapshots': settings['skip_dlm_snapshots'],
                  'skip_third_party_backup_snapshots': settings['skip_third_party_backup_snapshots'],
                  'third_party_backup_markers': settings['third_party_backup_markers'],
                  'ignore_tag_keys': settings['ignore_tag_keys'],
                  'ignore_tag_pairs': settings['ignore_tag_pairs'],
                  'max_snapshots_per_run': settings['max_snapshots_per_run'],
                  'excluded_regions': settings['excluded_regions'],
                  'regions': [],
                  'totals': {
                      'evaluated': 0,
                      'old_snapshots': 0,
                      'would_delete': 0,
                      'deleted': 0,
                      'skipped_too_new': 0,
                      'skipped_ami_associated': 0,
                      'skipped_aws_backup_managed': 0,
                      'skipped_dlm_managed': 0,
                      'skipped_third_party_backup_managed': 0,
                      'skipped_missing_approval_tag': 0,
                      'skipped_ignore_tag': 0,
                      'access_denied': 0,
                      'failed': 0
                  },
                  'region_results': []
              }

              try:
                  regions = get_enabled_regions(settings['excluded_regions'])
                  summary['regions'] = regions
                  for region in regions:
                      try:
                          region_result = scan_region(region, settings)
                      except Exception as e:
                          region_result = {
                              'region': region,
                              'evaluated': 0,
                              'old_snapshots': 0,
                              'would_delete': 0,
                              'deleted': 0,
                              'skipped_too_new': 0,
                              'skipped_ami_associated': 0,
                              'skipped_aws_backup_managed': 0,
                              'skipped_dlm_managed': 0,
                              'skipped_third_party_backup_managed': 0,
                              'skipped_missing_approval_tag': 0,
                              'skipped_ignore_tag': 0,
                              'access_denied': 0,
                              'failed': 1,
                              'failures': [{'error_code': type(e).__name__, 'error_message': str(e), 'trace': traceback.format_exc()}],
                              'candidates': [],
                              'deleted_snapshots': []
                          }
                      summary['region_results'].append(region_result)
                      for key in summary['totals']:
                          summary['totals'][key] += int(region_result.get(key, 0) or 0)
              finally:
                  summary['ended_at'] = utc_now_iso()
                  print(json.dumps(summary, indent=2, default=str))
              return summary

  RemediatorScheduleRule:
    Type: AWS::Events::Rule
    Condition: IsScheduleEnabled
    Properties:
      Name: !Sub '${ProjectName}-schedule'
      Description: Scheduled run for Old EBS Snapshot Cleanup Remediator.
      ScheduleExpression: !Ref ScheduleExpression
      State: ENABLED
      Targets:
        - Arn: !GetAtt RemediatorFunction.Arn
          Id: OldEbsSnapshotCleanupRemediatorTarget
          Input: '{"source":"eventbridge-schedule"}'

  RemediatorSchedulePermission:
    Type: AWS::Lambda::Permission
    Condition: IsScheduleEnabled
    Properties:
      FunctionName: !Ref RemediatorFunction
      Action: lambda:InvokeFunction
      Principal: events.amazonaws.com
      SourceArn: !GetAtt RemediatorScheduleRule.Arn

Outputs:
  RemediatorFunctionName:
    Description: Lambda function name.
    Value: !Ref RemediatorFunction

  RemediatorFunctionArn:
    Description: Lambda function ARN.
    Value: !GetAtt RemediatorFunction.Arn

  DryRunSetting:
    Description: Current DryRun setting.
    Value: !Ref DryRun

  MinimumSnapshotAgeDaysSetting:
    Description: Current minimum snapshot age setting.
    Value: !Ref MinimumSnapshotAgeDays

  RequireApprovalTagSetting:
    Description: Current RequireApprovalTag setting.
    Value: !Ref RequireApprovalTag

  SkipThirdPartyBackupSnapshotsSetting:
    Description: Current SkipThirdPartyBackupSnapshots setting.
    Value: !Ref SkipThirdPartyBackupSnapshots

  ManualDryRunExample:
    Description: Example manual dry-run invocation payload.
    Value: !Sub 'aws lambda invoke --function-name ${RemediatorFunction} --cli-binary-format raw-in-base64-out --payload ''{"dry_run":true}'' response.json'

  ManualBatchCleanupDryRunExample:
    Description: Example date-based batch cleanup dry-run payload.
    Value: !Sub 'aws lambda invoke --function-name ${RemediatorFunction} --cli-binary-format raw-in-base64-out --payload ''{"dry_run":true,"require_approval_tag":false,"minimum_snapshot_age_days":30,"max_snapshots_per_run":100}'' response.json'
